!pip install xgboost
!pip install seaborn
Looking in indexes: https://mirrors.cloud.aliyuncs.com/pypi/simple/
Collecting xgboost
  Downloading https://mirrors.cloud.aliyuncs.com/pypi/packages/a6/32/cd71fbcebde0809c4e911c9f5f65833c2a4bb43defc63d0b9377f3800bda/xgboost-1.6.2-py3-none-manylinux2014_x86_64.whl (255.9 MB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 255.9/255.9 MB 5.8 MB/s eta 0:00:0000:0100:01
Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from xgboost) (1.18.5)
Requirement already satisfied: scipy in /opt/conda/lib/python3.7/site-packages (from xgboost) (1.7.3)
Installing collected packages: xgboost
Successfully installed xgboost-1.6.2
WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv

[notice] A new release of pip available: 22.3.1 -> 24.0
[notice] To update, run: pip install --upgrade pip
Looking in indexes: https://mirrors.cloud.aliyuncs.com/pypi/simple/
Collecting seaborn
  Downloading https://mirrors.cloud.aliyuncs.com/pypi/packages/8f/2e/17bbb83fbf102687bb2aa3d808add39da820a7698159302a1a69bb82e01c/seaborn-0.12.2-py3-none-any.whl (293 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 293.3/293.3 kB 11.8 MB/s eta 0:00:00
Requirement already satisfied: typing_extensions in /opt/conda/lib/python3.7/site-packages (from seaborn) (4.1.1)
Requirement already satisfied: matplotlib!=3.6.1,>=3.1 in /opt/conda/lib/python3.7/site-packages (from seaborn) (3.5.3)
Requirement already satisfied: numpy!=1.24.0,>=1.17 in /opt/conda/lib/python3.7/site-packages (from seaborn) (1.18.5)
Requirement already satisfied: pandas>=0.25 in /opt/conda/lib/python3.7/site-packages (from seaborn) (1.3.5)
Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (21.3)
Requirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.7/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (0.11.0)
Requirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.7/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (2.8.2)
Requirement already satisfied: pyparsing>=2.2.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (3.0.9)
Requirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (1.4.4)
Requirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (4.38.0)
Requirement already satisfied: pillow>=6.2.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (9.3.0)
Requirement already satisfied: pytz>=2017.3 in /opt/conda/lib/python3.7/site-packages (from pandas>=0.25->seaborn) (2022.6)
Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil>=2.7->matplotlib!=3.6.1,>=3.1->seaborn) (1.16.0)
Installing collected packages: seaborn
Successfully installed seaborn-0.12.2
WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv

[notice] A new release of pip available: 22.3.1 -> 24.0
[notice] To update, run: pip install --upgrade pip
import joblib
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
​
import datetime
from tqdm import tqdm
from sklearn.preprocessing import LabelEncoder
from sklearn.svm import LinearSVC
from sklearn.preprocessing import PolynomialFeatures
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
import xgboost as xgb
from xgboost import XGBClassifier
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.feature_selection import chi2,SelectKBest,f_classif
from sklearn.preprocessing import MinMaxScaler
from sklearn.pipeline import Pipeline
from sklearn.model_selection import StratifiedKFold, KFold,cross_validate
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score, f1_score, roc_auc_score, log_loss,mean_absolute_error
​
import seaborn as sns
import warnings
​
warnings.filterwarnings('ignore')
data = pd.read_csv('/mnt/workspace/downloads/168012/train.csv')
data.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 800000 entries, 0 to 799999
Data columns (total 47 columns):
 #   Column              Non-Null Count   Dtype  
---  ------              --------------   -----  
 0   id                  800000 non-null  int64  
 1   loanAmnt            800000 non-null  float64
 2   term                800000 non-null  int64  
 3   interestRate        800000 non-null  float64
 4   installment         800000 non-null  float64
 5   grade               800000 non-null  object 
 6   subGrade            800000 non-null  object 
 7   employmentTitle     799999 non-null  float64
 8   employmentLength    753201 non-null  object 
 9   homeOwnership       800000 non-null  int64  
 10  annualIncome        800000 non-null  float64
 11  verificationStatus  800000 non-null  int64  
 12  issueDate           800000 non-null  object 
 13  isDefault           800000 non-null  int64  
 14  purpose             800000 non-null  int64  
 15  postCode            799999 non-null  float64
 16  regionCode          800000 non-null  int64  
 17  dti                 799761 non-null  float64
 18  delinquency_2years  800000 non-null  float64
 19  ficoRangeLow        800000 non-null  float64
 20  ficoRangeHigh       800000 non-null  float64
 21  openAcc             800000 non-null  float64
 22  pubRec              800000 non-null  float64
 23  pubRecBankruptcies  799595 non-null  float64
 24  revolBal            800000 non-null  float64
 25  revolUtil           799469 non-null  float64
 26  totalAcc            800000 non-null  float64
 27  initialListStatus   800000 non-null  int64  
 28  applicationType     800000 non-null  int64  
 29  earliesCreditLine   800000 non-null  object 
 30  title               799999 non-null  float64
 31  policyCode          800000 non-null  float64
 32  n0                  759730 non-null  float64
 33  n1                  759730 non-null  float64
 34  n2                  759730 non-null  float64
 35  n3                  759730 non-null  float64
 36  n4                  766761 non-null  float64
 37  n5                  759730 non-null  float64
 38  n6                  759730 non-null  float64
 39  n7                  759730 non-null  float64
 40  n8                  759729 non-null  float64
 41  n9                  759730 non-null  float64
 42  n10                 766761 non-null  float64
 43  n11                 730248 non-null  float64
 44  n12                 759730 non-null  float64
 45  n13                 759730 non-null  float64
 46  n14                 759730 non-null  float64
dtypes: float64(33), int64(9), object(5)
memory usage: 286.9+ MB
data.head()
id	loanAmnt	term	interestRate	installment	grade	subGrade	employmentTitle	employmentLength	homeOwnership	...	n5	n6	n7	n8	n9	n10	n11	n12	n13	n14
0	0	35000.0	5	19.52	917.97	E	E2	320.0	2 years	2	...	9.0	8.0	4.0	12.0	2.0	7.0	0.0	0.0	0.0	2.0
1	1	18000.0	5	18.49	461.90	D	D2	219843.0	5 years	0	...	NaN	NaN	NaN	NaN	NaN	13.0	NaN	NaN	NaN	NaN
2	2	12000.0	5	16.99	298.17	D	D3	31698.0	8 years	0	...	0.0	21.0	4.0	5.0	3.0	11.0	0.0	0.0	0.0	4.0
3	3	11000.0	3	7.26	340.96	A	A4	46854.0	10+ years	1	...	16.0	4.0	7.0	21.0	6.0	9.0	0.0	0.0	0.0	1.0
4	4	3000.0	3	12.99	101.07	C	C2	54.0	NaN	1	...	4.0	9.0	10.0	15.0	7.0	12.0	0.0	0.0	0.0	4.0
5 rows × 47 columns

data.describe().T
count	mean	std	min	25%	50%	75%	max
id	800000.0	399999.500000	230940.252015	0.00	199999.75	399999.500	599999.25	799999.00
loanAmnt	800000.0	14416.818875	8716.086178	500.00	8000.00	12000.000	20000.00	40000.00
term	800000.0	3.482745	0.855832	3.00	3.00	3.000	3.00	5.00
interestRate	800000.0	13.238391	4.765757	5.31	9.75	12.740	15.99	30.99
installment	800000.0	437.947723	261.460393	15.69	248.45	375.135	580.71	1715.42
employmentTitle	799999.0	72005.351714	106585.640204	0.00	427.00	7755.000	117663.50	378351.00
homeOwnership	800000.0	0.614213	0.675749	0.00	0.00	1.000	1.00	5.00
annualIncome	800000.0	76133.910493	68947.513672	0.00	45600.00	65000.000	90000.00	10999200.00
verificationStatus	800000.0	1.009683	0.782716	0.00	0.00	1.000	2.00	2.00
isDefault	800000.0	0.199513	0.399634	0.00	0.00	0.000	0.00	1.00
purpose	800000.0	1.745982	2.367453	0.00	0.00	0.000	4.00	13.00
postCode	799999.0	258.535648	200.037446	0.00	103.00	203.000	395.00	940.00
regionCode	800000.0	16.385758	11.036679	0.00	8.00	14.000	22.00	50.00
dti	799761.0	18.284557	11.150155	-1.00	11.79	17.610	24.06	999.00
delinquency_2years	800000.0	0.318239	0.880325	0.00	0.00	0.000	0.00	39.00
ficoRangeLow	800000.0	696.204081	31.865995	630.00	670.00	690.000	710.00	845.00
ficoRangeHigh	800000.0	700.204226	31.866674	634.00	674.00	694.000	714.00	850.00
openAcc	800000.0	11.598020	5.475286	0.00	8.00	11.000	14.00	86.00
pubRec	800000.0	0.214915	0.606467	0.00	0.00	0.000	0.00	86.00
pubRecBankruptcies	799595.0	0.134163	0.377471	0.00	0.00	0.000	0.00	12.00
revolBal	800000.0	16228.706505	22458.020544	0.00	5944.00	11132.000	19734.00	2904836.00
revolUtil	799469.0	51.790734	24.516126	0.00	33.40	52.100	70.70	892.30
totalAcc	800000.0	24.998861	11.999201	2.00	16.00	23.000	32.00	162.00
initialListStatus	800000.0	0.416953	0.493055	0.00	0.00	0.000	1.00	1.00
applicationType	800000.0	0.019267	0.137464	0.00	0.00	0.000	0.00	1.00
title	799999.0	1754.113589	7941.474040	0.00	0.00	1.000	5.00	61680.00
policyCode	800000.0	1.000000	0.000000	1.00	1.00	1.000	1.00	1.00
n0	759730.0	0.511932	1.333266	0.00	0.00	0.000	0.00	51.00
n1	759730.0	3.642330	2.246825	0.00	2.00	3.000	5.00	33.00
n2	759730.0	5.642648	3.302810	0.00	3.00	5.000	7.00	63.00
n3	759730.0	5.642648	3.302810	0.00	3.00	5.000	7.00	63.00
n4	766761.0	4.735641	2.949969	0.00	3.00	4.000	6.00	49.00
n5	759730.0	8.107937	4.799210	0.00	5.00	7.000	11.00	70.00
n6	759730.0	8.575994	7.400536	0.00	4.00	7.000	11.00	132.00
n7	759730.0	8.282953	4.561689	0.00	5.00	7.000	10.00	79.00
n8	759729.0	14.622488	8.124610	1.00	9.00	13.000	19.00	128.00
n9	759730.0	5.592345	3.216184	0.00	3.00	5.000	7.00	45.00
n10	766761.0	11.643896	5.484104	0.00	8.00	11.000	14.00	82.00
n11	730248.0	0.000815	0.030075	0.00	0.00	0.000	0.00	4.00
n12	759730.0	0.003384	0.062041	0.00	0.00	0.000	0.00	4.00
n13	759730.0	0.089366	0.509069	0.00	0.00	0.000	0.00	39.00
n14	759730.0	2.178606	1.844377	0.00	1.00	2.000	3.00	30.00
EDA
缺失值情况
#缺失值比例
data.isnull().sum()[data.isnull().sum()>0]*100/data.shape[0]
employmentTitle       0.000125
employmentLength      5.849875
postCode              0.000125
dti                   0.029875
pubRecBankruptcies    0.050625
revolUtil             0.066375
title                 0.000125
n0                    5.033750
n1                    5.033750
n2                    5.033750
n3                    5.033750
n4                    4.154875
n5                    5.033750
n6                    5.033750
n7                    5.033750
n8                    5.033875
n9                    5.033750
n10                   4.154875
n11                   8.719000
n12                   5.033750
n13                   5.033750
n14                   5.033750
dtype: float64
数值数据分析
num_data = data.select_dtypes(exclude=['object'])
num_fea = num_data.columns
num_fea
Index(['id', 'loanAmnt', 'term', 'interestRate', 'installment',
       'employmentTitle', 'homeOwnership', 'annualIncome',
       'verificationStatus', 'isDefault', 'purpose', 'postCode', 'regionCode',
       'dti', 'delinquency_2years', 'ficoRangeLow', 'ficoRangeHigh', 'openAcc',
       'pubRec', 'pubRecBankruptcies', 'revolBal', 'revolUtil', 'totalAcc',
       'initialListStatus', 'applicationType', 'title', 'policyCode', 'n0',
       'n1', 'n2', 'n3', 'n4', 'n5', 'n6', 'n7', 'n8', 'n9', 'n10', 'n11',
       'n12', 'n13', 'n14'],
      dtype='object')
由于数值型变量包括连续型变量和离散型变量，我们要把他们区分出来

series_fea = []
noserial_fea = []
for i in num_data.columns:
    if len(data[i].unique())<10:#以10作为区分值
        noserial_fea.append(i)
    else:
        series_fea.append(i)
print('series_fea:',series_fea,'\n','noserial_fea:',noserial_fea)
series_fea: ['id', 'loanAmnt', 'interestRate', 'installment', 'employmentTitle', 'annualIncome', 'purpose', 'postCode', 'regionCode', 'dti', 'delinquency_2years', 'ficoRangeLow', 'ficoRangeHigh', 'openAcc', 'pubRec', 'pubRecBankruptcies', 'revolBal', 'revolUtil', 'totalAcc', 'title', 'n0', 'n1', 'n2', 'n3', 'n4', 'n5', 'n6', 'n7', 'n8', 'n9', 'n10', 'n13', 'n14'] 
 noserial_fea: ['term', 'homeOwnership', 'verificationStatus', 'isDefault', 'initialListStatus', 'applicationType', 'policyCode', 'n11', 'n12']
离散数值变量分析
for i in noserial_fea:
    print(data[i].value_counts())
    print(("*")*20)
3    606902
5    193098
Name: term, dtype: int64
********************
0    395732
1    317660
2     86309
3       185
5        81
4        33
Name: homeOwnership, dtype: int64
********************
1    309810
2    248968
0    241222
Name: verificationStatus, dtype: int64
********************
0    640390
1    159610
Name: isDefault, dtype: int64
********************
0    466438
1    333562
Name: initialListStatus, dtype: int64
********************
0    784586
1     15414
Name: applicationType, dtype: int64
********************
1.0    800000
Name: policyCode, dtype: int64
********************
0.0    729682
1.0       540
2.0        24
4.0         1
3.0         1
Name: n11, dtype: int64
********************
0.0    757315
1.0      2281
2.0       115
3.0        16
4.0         3
Name: n12, dtype: int64
********************
policyCode只有一个值，对模型无用，随后可删除

连续数值变量分析
series_fea
['id',
 'loanAmnt',
 'interestRate',
 'installment',
 'employmentTitle',
 'annualIncome',
 'purpose',
 'postCode',
 'regionCode',
 'dti',
 'delinquency_2years',
 'ficoRangeLow',
 'ficoRangeHigh',
 'openAcc',
 'pubRec',
 'pubRecBankruptcies',
 'revolBal',
 'revolUtil',
 'totalAcc',
 'title',
 'n0',
 'n1',
 'n2',
 'n3',
 'n4',
 'n5',
 'n6',
 'n7',
 'n8',
 'n9',
 'n10',
 'n13',
 'n14']
for col in series_fea:
    sns.set_theme()
    sns.relplot(data=data,x='issueDate',y=col,hue='isDefault',kind='line')
    plt.show()

































从上面图片可以看出来，按时间顺序违约与不违约差距较大的有以下几个字段： [ 'loanAmnt', 'interestRate', 'installment', 'issueDate', 'purpose', 'dti', 'delinquency_2years', 'ficoRangeLow', 'ficoRangeHigh', 'revolUtil', 'n1', 'n2', 'n3', 'n7', 'n9', 'n10', 'n14']，同时我们可以发现，对于除匿名特征之外的数据从2012年开始数据才会出现一些规律性，我们有理由认为，2012年前的数据是脏数据，随后可以尝试剔除。

f = pd.melt(data, value_vars=series_fea[1:])
g = sns.FacetGrid(f, col="variable",  col_wrap=2, sharex=False, sharey=False)
g = g.map(sns.distplot, "value")

大部分数据都是右偏的，模型处理的时候可以将其标准化

类别数据分析
category_data = data.select_dtypes(include=['object'])
category_fea = category_data.columns
category_fea
Index(['grade', 'subGrade', 'employmentLength', 'issueDate',
       'earliesCreditLine'],
      dtype='object')
data['grade'].value_counts()
B    233690
C    227118
A    139661
D    119453
E     55661
F     19053
G      5364
Name: grade, dtype: int64
fig, ax =plt.subplots(2,2,constrained_layout=True, figsize=(20, 20))
for i in range(2):
    for j in range(2):
        axesSub = sns.barplot(data[category_fea[i+j]].value_counts(), ax=ax[i][j])
        axesSub.set_title(category_fea[i+j])

特征工程
时间数据特征构建
import datetime
data['issueDate'] = pd.to_datetime(data['issueDate'],format='%Y-%m-%d')
startdate = datetime.datetime.strptime('2007-06-01', '%Y-%m-%d')
data['issueDateDT'] = data['issueDate'].apply(lambda x: x-startdate).dt.days
考虑到有离散数值也有连续数值，因此连续数值使用中位数来填充缺失值，离散数值用众数填充缺失值

缺失数据填充
#按照中位数填充数值型特征
data[num_fea] = data[num_fea].fillna(data[num_fea].median())
#按照众数填充类别型特征
data[category_fea] = data[category_fea].fillna(data[category_fea].mode())
data.isnull().sum()
id                        0
loanAmnt                  0
term                      0
interestRate              0
installment               0
grade                     0
subGrade                  0
employmentTitle           0
employmentLength      46799
homeOwnership             0
annualIncome              0
verificationStatus        0
issueDate                 0
isDefault                 0
purpose                   0
postCode                  0
regionCode                0
dti                       0
delinquency_2years        0
ficoRangeLow              0
ficoRangeHigh             0
openAcc                   0
pubRec                    0
pubRecBankruptcies        0
revolBal                  0
revolUtil                 0
totalAcc                  0
initialListStatus         0
applicationType           0
earliesCreditLine         0
title                     0
policyCode                0
n0                        0
n1                        0
n2                        0
n3                        0
n4                        0
n5                        0
n6                        0
n7                        0
n8                        0
n9                        0
n10                       0
n11                       0
n12                       0
n13                       0
n14                       0
issueDateDT               0
dtype: int64
data['employmentLength'].value_counts()
10+ years    262753
2 years       72358
< 1 year      64237
3 years       64152
1 year        52489
5 years       50102
4 years       47985
6 years       37254
8 years       36192
7 years       35407
9 years       30272
Name: employmentLength, dtype: int64
#在类别数据分析时，employmentLength字段众数是10+ years，因此将其填充缺失值
data['employmentLength'] = data['employmentLength'].fillna('10+ years')
data.isnull().sum()
id                    0
loanAmnt              0
term                  0
interestRate          0
installment           0
grade                 0
subGrade              0
employmentTitle       0
employmentLength      0
homeOwnership         0
annualIncome          0
verificationStatus    0
issueDate             0
isDefault             0
purpose               0
postCode              0
regionCode            0
dti                   0
delinquency_2years    0
ficoRangeLow          0
ficoRangeHigh         0
openAcc               0
pubRec                0
pubRecBankruptcies    0
revolBal              0
revolUtil             0
totalAcc              0
initialListStatus     0
applicationType       0
earliesCreditLine     0
title                 0
policyCode            0
n0                    0
n1                    0
n2                    0
n3                    0
n4                    0
n5                    0
n6                    0
n7                    0
n8                    0
n9                    0
n10                   0
n11                   0
n12                   0
n13                   0
n14                   0
issueDateDT           0
dtype: int64
类别特征处理（编码等）
category_fea
Index(['grade', 'subGrade', 'employmentLength', 'issueDate',
       'earliesCreditLine'],
      dtype='object')
剩余几个类别数据，grade：贷款等级、subGrade：贷款等级之子级、employmentLength：就业年限（年）、earliesCreditLine：借款人最早报告的信用额度开立的月份

贷款等级处理
def deal_subgrade(data):
    # 由于subGrade有等级区分，因此将其数字化，英文字母作为10位，后面内容作为个位进行拼接，将此object转为int格式
    return data.apply(lambda x:x[0]).map({'A':1,'B':2,'C':3,'D':4,'E':5,'F':6,'G':7})*10+data.apply(lambda x:x[1]).astype(int)
data['subGrade'] = deal_subgrade(data['subGrade'])
def deal_grade(data):   
    # grade标签化处理
    Le = LabelEncoder()
    result = Le.fit_transform(data)
    joblib.dump(Le,'grade_label_encoder.model')
    return result
data['grade'] = deal_grade(data['grade'])
就业年限处理
#由于就业年限具有顺序性，同样将其转换为int格式的数据
data['employmentLength'] = data['employmentLength'].map({'< 1 year':0,'1 years':1,'2 years':2,'3 years':3,'4 years':4,'5 years':5,'6 years':6,'7 years':7
               ,'8 years':8,'9 years':9,'10+ years':10})
data['employmentLength'].value_counts()
10.0    309552
2.0      72358
0.0      64237
3.0      64152
5.0      50102
4.0      47985
6.0      37254
8.0      36192
7.0      35407
9.0      30272
Name: employmentLength, dtype: int64
data['employmentLength']
0          2.0
1          5.0
2          8.0
3         10.0
4         10.0
          ... 
799995     7.0
799996    10.0
799997    10.0
799998    10.0
799999     5.0
Name: employmentLength, Length: 800000, dtype: float64
借款人最早报告的信用额度开立的月份处理
# 借款人最最早报告的信用额度开立的月份与其贷款发放的月份做差进行特征构建
data['earliesCreditLine'] = pd.to_datetime(data['earliesCreditLine'],format='%b-%Y')
data['earliesCreditLineDT'] = (data['issueDate']-data['earliesCreditLine']).dt.days
缺失值和类别特征已经处理完成，接下来可以进行异常值的剔除。

data.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 800000 entries, 0 to 799999
Data columns (total 49 columns):
 #   Column               Non-Null Count   Dtype         
---  ------               --------------   -----         
 0   id                   800000 non-null  int64         
 1   loanAmnt             800000 non-null  float64       
 2   term                 800000 non-null  int64         
 3   interestRate         800000 non-null  float64       
 4   installment          800000 non-null  float64       
 5   grade                800000 non-null  int64         
 6   subGrade             800000 non-null  int64         
 7   employmentTitle      800000 non-null  float64       
 8   employmentLength     747511 non-null  float64       
 9   homeOwnership        800000 non-null  int64         
 10  annualIncome         800000 non-null  float64       
 11  verificationStatus   800000 non-null  int64         
 12  issueDate            800000 non-null  datetime64[ns]
 13  isDefault            800000 non-null  int64         
 14  purpose              800000 non-null  int64         
 15  postCode             800000 non-null  float64       
 16  regionCode           800000 non-null  int64         
 17  dti                  800000 non-null  float64       
 18  delinquency_2years   800000 non-null  float64       
 19  ficoRangeLow         800000 non-null  float64       
 20  ficoRangeHigh        800000 non-null  float64       
 21  openAcc              800000 non-null  float64       
 22  pubRec               800000 non-null  float64       
 23  pubRecBankruptcies   800000 non-null  float64       
 24  revolBal             800000 non-null  float64       
 25  revolUtil            800000 non-null  float64       
 26  totalAcc             800000 non-null  float64       
 27  initialListStatus    800000 non-null  int64         
 28  applicationType      800000 non-null  int64         
 29  earliesCreditLine    800000 non-null  datetime64[ns]
 30  title                800000 non-null  float64       
 31  policyCode           800000 non-null  float64       
 32  n0                   800000 non-null  float64       
 33  n1                   800000 non-null  float64       
 34  n2                   800000 non-null  float64       
 35  n3                   800000 non-null  float64       
 36  n4                   800000 non-null  float64       
 37  n5                   800000 non-null  float64       
 38  n6                   800000 non-null  float64       
 39  n7                   800000 non-null  float64       
 40  n8                   800000 non-null  float64       
 41  n9                   800000 non-null  float64       
 42  n10                  800000 non-null  float64       
 43  n11                  800000 non-null  float64       
 44  n12                  800000 non-null  float64       
 45  n13                  800000 non-null  float64       
 46  n14                  800000 non-null  float64       
 47  issueDateDT          800000 non-null  int64         
 48  earliesCreditLineDT  800000 non-null  int64         
dtypes: datetime64[ns](2), float64(34), int64(13)
memory usage: 299.1 MB
异常值剔除
在异常值剔除之前，我们先看看时间的分布

sns.set_theme()
​
sns.displot(data,x='issueDate')
<seaborn.axisgrid.FacetGrid at 0x7f9c93a87a90>

由于在连续数值变量分析时，发现2012年前的数据存在异常情况，而且占比不大，因此直接将这些数据进行删除

train_data = data.loc[data['issueDate']>='2012',:]
def find_outliers_by_3segama(data,fea):
    data_std = np.std(data[fea])
    data_mean = np.mean(data[fea])
    outliers_cut_off = data_std * 3
    lower_rule = data_mean - outliers_cut_off
    upper_rule = data_mean + outliers_cut_off
    data[fea+'_outliers'] = data[fea].apply(lambda x:str('异常值') if x > upper_rule or x < lower_rule else '正常值')
    return data
for fea in num_fea:
    data = find_outliers_by_3segama(train_data,fea)
    print(data[fea+'_outliers'].value_counts())
    print(data.groupby(fea+'_outliers')['isDefault'].sum())
    print('*'*10)
正常值    776370
Name: id_outliers, dtype: int64
id_outliers
正常值    156304
Name: isDefault, dtype: int64
**********
正常值    776370
Name: loanAmnt_outliers, dtype: int64
loanAmnt_outliers
正常值    156304
Name: isDefault, dtype: int64
**********
正常值    776370
Name: term_outliers, dtype: int64
term_outliers
正常值    156304
Name: isDefault, dtype: int64
**********
正常值    770629
异常值      5741
Name: interestRate_outliers, dtype: int64
interestRate_outliers
异常值      2916
正常值    153388
Name: isDefault, dtype: int64
**********
正常值    769069
异常值      7301
Name: installment_outliers, dtype: int64
installment_outliers
异常值      2011
正常值    154293
Name: isDefault, dtype: int64
**********
正常值    776370
Name: employmentTitle_outliers, dtype: int64
employmentTitle_outliers
正常值    156304
Name: isDefault, dtype: int64
**********
正常值    776127
异常值       243
Name: homeOwnership_outliers, dtype: int64
homeOwnership_outliers
异常值        49
正常值    156255
Name: isDefault, dtype: int64
**********
正常值    770502
异常值      5868
Name: annualIncome_outliers, dtype: int64
annualIncome_outliers
异常值       741
正常值    155563
Name: isDefault, dtype: int64
**********
正常值    776370
Name: verificationStatus_outliers, dtype: int64
verificationStatus_outliers
正常值    156304
Name: isDefault, dtype: int64
**********
正常值    776370
Name: isDefault_outliers, dtype: int64
isDefault_outliers
正常值    156304
Name: isDefault, dtype: int64
**********
正常值    760940
异常值     15430
Name: purpose_outliers, dtype: int64
purpose_outliers
异常值      3425
正常值    152879
Name: isDefault, dtype: int64
**********
正常值    775351
异常值      1019
Name: postCode_outliers, dtype: int64
postCode_outliers
异常值       212
正常值    156092
Name: isDefault, dtype: int64
**********
正常值    776368
异常值         2
Name: regionCode_outliers, dtype: int64
regionCode_outliers
异常值         1
正常值    156303
Name: isDefault, dtype: int64
**********
正常值    774842
异常值      1528
Name: dti_outliers, dtype: int64
dti_outliers
异常值       455
正常值    155849
Name: isDefault, dtype: int64
**********
正常值    754807
异常值     21563
Name: delinquency_2years_outliers, dtype: int64
delinquency_2years_outliers
异常值      5060
正常值    151244
Name: isDefault, dtype: int64
**********
正常值    765187
异常值     11183
Name: ficoRangeLow_outliers, dtype: int64
ficoRangeLow_outliers
异常值       746
正常值    155558
Name: isDefault, dtype: int64
**********
正常值    765187
异常值     11183
Name: ficoRangeHigh_outliers, dtype: int64
ficoRangeHigh_outliers
异常值       746
正常值    155558
Name: isDefault, dtype: int64
**********
正常值    767288
异常值      9082
Name: openAcc_outliers, dtype: int64
openAcc_outliers
异常值      2188
正常值    154116
Name: isDefault, dtype: int64
**********
正常值    768847
异常值      7523
Name: pubRec_outliers, dtype: int64
pubRec_outliers
异常值      1701
正常值    154603
Name: isDefault, dtype: int64
**********
正常值    770494
异常值      5876
Name: pubRecBankruptcies_outliers, dtype: int64
pubRecBankruptcies_outliers
异常值      1421
正常值    154883
Name: isDefault, dtype: int64
**********
正常值    766730
异常值      9640
Name: revolBal_outliers, dtype: int64
revolBal_outliers
异常值      1307
正常值    154997
Name: isDefault, dtype: int64
**********
正常值    776316
异常值        54
Name: revolUtil_outliers, dtype: int64
revolUtil_outliers
异常值        23
正常值    156281
Name: isDefault, dtype: int64
**********
正常值    769151
异常值      7219
Name: totalAcc_outliers, dtype: int64
totalAcc_outliers
异常值      1461
正常值    154843
Name: isDefault, dtype: int64
**********
正常值    776370
Name: initialListStatus_outliers, dtype: int64
initialListStatus_outliers
正常值    156304
Name: isDefault, dtype: int64
**********
正常值    760956
异常值     15414
Name: applicationType_outliers, dtype: int64
applicationType_outliers
异常值      3875
正常值    152429
Name: isDefault, dtype: int64
**********
正常值    751375
异常值     24995
Name: title_outliers, dtype: int64
title_outliers
异常值      3919
正常值    152385
Name: isDefault, dtype: int64
**********
正常值    776370
Name: policyCode_outliers, dtype: int64
policyCode_outliers
正常值    156304
Name: isDefault, dtype: int64
**********
正常值    759143
异常值     17227
Name: n0_outliers, dtype: int64
n0_outliers
异常值      3485
正常值    152819
Name: isDefault, dtype: int64
**********
正常值    766870
异常值      9500
Name: n1_outliers, dtype: int64
n1_outliers
异常值      2491
正常值    153813
Name: isDefault, dtype: int64
**********
正常值    765437
异常值     10933
Name: n2_outliers, dtype: int64
n2_outliers
异常值      3205
正常值    153099
Name: isDefault, dtype: int64
**********
正常值    765437
异常值     10933
Name: n3_outliers, dtype: int64
n3_outliers
异常值      3205
正常值    153099
Name: isDefault, dtype: int64
**********
正常值    765030
异常值     11340
Name: n4_outliers, dtype: int64
n4_outliers
异常值      2476
正常值    153828
Name: isDefault, dtype: int64
**********
正常值    766725
异常值      9645
Name: n5_outliers, dtype: int64
n5_outliers
异常值      1858
正常值    154446
Name: isDefault, dtype: int64
**********
正常值    762376
异常值     13994
Name: n6_outliers, dtype: int64
n6_outliers
异常值      3182
正常值    153122
Name: isDefault, dtype: int64
**********
正常值    764800
异常值     11570
Name: n7_outliers, dtype: int64
n7_outliers
异常值      2746
正常值    153558
Name: isDefault, dtype: int64
**********
正常值    765995
异常值     10375
Name: n8_outliers, dtype: int64
n8_outliers
异常值      2131
正常值    154173
Name: isDefault, dtype: int64
**********
正常值    766820
异常值      9550
Name: n9_outliers, dtype: int64
n9_outliers
异常值      2842
正常值    153462
Name: isDefault, dtype: int64
**********
正常值    765349
异常值     11021
Name: n10_outliers, dtype: int64
n10_outliers
异常值      2639
正常值    153665
Name: isDefault, dtype: int64
**********
正常值    775804
异常值       566
Name: n11_outliers, dtype: int64
n11_outliers
异常值       112
正常值    156192
Name: isDefault, dtype: int64
**********
正常值    773955
异常值      2415
Name: n12_outliers, dtype: int64
n12_outliers
异常值       545
正常值    155759
Name: isDefault, dtype: int64
**********
正常值    765277
异常值     11093
Name: n13_outliers, dtype: int64
n13_outliers
异常值      2482
正常值    153822
Name: isDefault, dtype: int64
**********
正常值    765254
异常值     11116
Name: n14_outliers, dtype: int64
n14_outliers
异常值      3364
正常值    152940
Name: isDefault, dtype: int64
**********
#删除异常值
for fea in num_fea:
    train_data = train_data[train_data[fea+'_outliers']=='正常值']
    train_data = train_data.reset_index(drop=True)
train_data.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 594068 entries, 0 to 594067
Data columns (total 91 columns):
 #   Column                       Non-Null Count   Dtype         
---  ------                       --------------   -----         
 0   id                           594068 non-null  int64         
 1   loanAmnt                     594068 non-null  float64       
 2   term                         594068 non-null  int64         
 3   interestRate                 594068 non-null  float64       
 4   installment                  594068 non-null  float64       
 5   grade                        594068 non-null  int64         
 6   subGrade                     594068 non-null  int64         
 7   employmentTitle              594068 non-null  float64       
 8   employmentLength             554411 non-null  float64       
 9   homeOwnership                594068 non-null  int64         
 10  annualIncome                 594068 non-null  float64       
 11  verificationStatus           594068 non-null  int64         
 12  issueDate                    594068 non-null  datetime64[ns]
 13  isDefault                    594068 non-null  int64         
 14  purpose                      594068 non-null  int64         
 15  postCode                     594068 non-null  float64       
 16  regionCode                   594068 non-null  int64         
 17  dti                          594068 non-null  float64       
 18  delinquency_2years           594068 non-null  float64       
 19  ficoRangeLow                 594068 non-null  float64       
 20  ficoRangeHigh                594068 non-null  float64       
 21  openAcc                      594068 non-null  float64       
 22  pubRec                       594068 non-null  float64       
 23  pubRecBankruptcies           594068 non-null  float64       
 24  revolBal                     594068 non-null  float64       
 25  revolUtil                    594068 non-null  float64       
 26  totalAcc                     594068 non-null  float64       
 27  initialListStatus            594068 non-null  int64         
 28  applicationType              594068 non-null  int64         
 29  earliesCreditLine            594068 non-null  datetime64[ns]
 30  title                        594068 non-null  float64       
 31  policyCode                   594068 non-null  float64       
 32  n0                           594068 non-null  float64       
 33  n1                           594068 non-null  float64       
 34  n2                           594068 non-null  float64       
 35  n3                           594068 non-null  float64       
 36  n4                           594068 non-null  float64       
 37  n5                           594068 non-null  float64       
 38  n6                           594068 non-null  float64       
 39  n7                           594068 non-null  float64       
 40  n8                           594068 non-null  float64       
 41  n9                           594068 non-null  float64       
 42  n10                          594068 non-null  float64       
 43  n11                          594068 non-null  float64       
 44  n12                          594068 non-null  float64       
 45  n13                          594068 non-null  float64       
 46  n14                          594068 non-null  float64       
 47  issueDateDT                  594068 non-null  int64         
 48  earliesCreditLineDT          594068 non-null  int64         
 49  id_outliers                  594068 non-null  object        
 50  loanAmnt_outliers            594068 non-null  object        
 51  term_outliers                594068 non-null  object        
 52  interestRate_outliers        594068 non-null  object        
 53  installment_outliers         594068 non-null  object        
 54  employmentTitle_outliers     594068 non-null  object        
 55  homeOwnership_outliers       594068 non-null  object        
 56  annualIncome_outliers        594068 non-null  object        
 57  verificationStatus_outliers  594068 non-null  object        
 58  isDefault_outliers           594068 non-null  object        
 59  purpose_outliers             594068 non-null  object        
 60  postCode_outliers            594068 non-null  object        
 61  regionCode_outliers          594068 non-null  object        
 62  dti_outliers                 594068 non-null  object        
 63  delinquency_2years_outliers  594068 non-null  object        
 64  ficoRangeLow_outliers        594068 non-null  object        
 65  ficoRangeHigh_outliers       594068 non-null  object        
 66  openAcc_outliers             594068 non-null  object        
 67  pubRec_outliers              594068 non-null  object        
 68  pubRecBankruptcies_outliers  594068 non-null  object        
 69  revolBal_outliers            594068 non-null  object        
 70  revolUtil_outliers           594068 non-null  object        
 71  totalAcc_outliers            594068 non-null  object        
 72  initialListStatus_outliers   594068 non-null  object        
 73  applicationType_outliers     594068 non-null  object        
 74  title_outliers               594068 non-null  object        
 75  policyCode_outliers          594068 non-null  object        
 76  n0_outliers                  594068 non-null  object        
 77  n1_outliers                  594068 non-null  object        
 78  n2_outliers                  594068 non-null  object        
 79  n3_outliers                  594068 non-null  object        
 80  n4_outliers                  594068 non-null  object        
 81  n5_outliers                  594068 non-null  object        
 82  n6_outliers                  594068 non-null  object        
 83  n7_outliers                  594068 non-null  object        
 84  n8_outliers                  594068 non-null  object        
 85  n9_outliers                  594068 non-null  object        
 86  n10_outliers                 594068 non-null  object        
 87  n11_outliers                 594068 non-null  object        
 88  n12_outliers                 594068 non-null  object        
 89  n13_outliers                 594068 non-null  object        
 90  n14_outliers                 594068 non-null  object        
dtypes: datetime64[ns](2), float64(34), int64(13), object(42)
memory usage: 412.4+ MB
train_data['employmentLength'] = train_data['employmentLength'].fillna(train_data['employmentLength'].median())
train_data.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 594068 entries, 0 to 594067
Data columns (total 91 columns):
 #   Column                       Non-Null Count   Dtype         
---  ------                       --------------   -----         
 0   id                           594068 non-null  int64         
 1   loanAmnt                     594068 non-null  float64       
 2   term                         594068 non-null  int64         
 3   interestRate                 594068 non-null  float64       
 4   installment                  594068 non-null  float64       
 5   grade                        594068 non-null  int64         
 6   subGrade                     594068 non-null  int64         
 7   employmentTitle              594068 non-null  float64       
 8   employmentLength             594068 non-null  float64       
 9   homeOwnership                594068 non-null  int64         
 10  annualIncome                 594068 non-null  float64       
 11  verificationStatus           594068 non-null  int64         
 12  issueDate                    594068 non-null  datetime64[ns]
 13  isDefault                    594068 non-null  int64         
 14  purpose                      594068 non-null  int64         
 15  postCode                     594068 non-null  float64       
 16  regionCode                   594068 non-null  int64         
 17  dti                          594068 non-null  float64       
 18  delinquency_2years           594068 non-null  float64       
 19  ficoRangeLow                 594068 non-null  float64       
 20  ficoRangeHigh                594068 non-null  float64       
 21  openAcc                      594068 non-null  float64       
 22  pubRec                       594068 non-null  float64       
 23  pubRecBankruptcies           594068 non-null  float64       
 24  revolBal                     594068 non-null  float64       
 25  revolUtil                    594068 non-null  float64       
 26  totalAcc                     594068 non-null  float64       
 27  initialListStatus            594068 non-null  int64         
 28  applicationType              594068 non-null  int64         
 29  earliesCreditLine            594068 non-null  datetime64[ns]
 30  title                        594068 non-null  float64       
 31  policyCode                   594068 non-null  float64       
 32  n0                           594068 non-null  float64       
 33  n1                           594068 non-null  float64       
 34  n2                           594068 non-null  float64       
 35  n3                           594068 non-null  float64       
 36  n4                           594068 non-null  float64       
 37  n5                           594068 non-null  float64       
 38  n6                           594068 non-null  float64       
 39  n7                           594068 non-null  float64       
 40  n8                           594068 non-null  float64       
 41  n9                           594068 non-null  float64       
 42  n10                          594068 non-null  float64       
 43  n11                          594068 non-null  float64       
 44  n12                          594068 non-null  float64       
 45  n13                          594068 non-null  float64       
 46  n14                          594068 non-null  float64       
 47  issueDateDT                  594068 non-null  int64         
 48  earliesCreditLineDT          594068 non-null  int64         
 49  id_outliers                  594068 non-null  object        
 50  loanAmnt_outliers            594068 non-null  object        
 51  term_outliers                594068 non-null  object        
 52  interestRate_outliers        594068 non-null  object        
 53  installment_outliers         594068 non-null  object        
 54  employmentTitle_outliers     594068 non-null  object        
 55  homeOwnership_outliers       594068 non-null  object        
 56  annualIncome_outliers        594068 non-null  object        
 57  verificationStatus_outliers  594068 non-null  object        
 58  isDefault_outliers           594068 non-null  object        
 59  purpose_outliers             594068 non-null  object        
 60  postCode_outliers            594068 non-null  object        
 61  regionCode_outliers          594068 non-null  object        
 62  dti_outliers                 594068 non-null  object        
 63  delinquency_2years_outliers  594068 non-null  object        
 64  ficoRangeLow_outliers        594068 non-null  object        
 65  ficoRangeHigh_outliers       594068 non-null  object        
 66  openAcc_outliers             594068 non-null  object        
 67  pubRec_outliers              594068 non-null  object        
 68  pubRecBankruptcies_outliers  594068 non-null  object        
 69  revolBal_outliers            594068 non-null  object        
 70  revolUtil_outliers           594068 non-null  object        
 71  totalAcc_outliers            594068 non-null  object        
 72  initialListStatus_outliers   594068 non-null  object        
 73  applicationType_outliers     594068 non-null  object        
 74  title_outliers               594068 non-null  object        
 75  policyCode_outliers          594068 non-null  object        
 76  n0_outliers                  594068 non-null  object        
 77  n1_outliers                  594068 non-null  object        
 78  n2_outliers                  594068 non-null  object        
 79  n3_outliers                  594068 non-null  object        
 80  n4_outliers                  594068 non-null  object        
 81  n5_outliers                  594068 non-null  object        
 82  n6_outliers                  594068 non-null  object        
 83  n7_outliers                  594068 non-null  object        
 84  n8_outliers                  594068 non-null  object        
 85  n9_outliers                  594068 non-null  object        
 86  n10_outliers                 594068 non-null  object        
 87  n11_outliers                 594068 non-null  object        
 88  n12_outliers                 594068 non-null  object        
 89  n13_outliers                 594068 non-null  object        
 90  n14_outliers                 594068 non-null  object        
dtypes: datetime64[ns](2), float64(34), int64(13), object(42)
memory usage: 412.4+ MB
#观察相关性
def corr_desc(data):
    corrmat = data.corr().T
    corrmat = abs(corrmat)
    f,ax=plt.subplots(figsize=(40,18))
    sns.heatmap(corrmat,vmax=0.8,square=True,cmap="YlGnBu",annot=True)
    plt.savefig('corr_heatmap.jpg')
corr_desc(train_data)

筛选特征进行数据分箱
data['subGrade'].astype(str).values
array(['52', '42', '43', ..., '33', '14', '23'], dtype=object)
特征选择
train_data = train_data.drop(columns=['id','issueDate','earliesCreditLine'])
train_data = train_data.select_dtypes(exclude='object')
train_data.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 594068 entries, 0 to 594067
Data columns (total 46 columns):
 #   Column               Non-Null Count   Dtype  
---  ------               --------------   -----  
 0   loanAmnt             594068 non-null  float64
 1   term                 594068 non-null  int64  
 2   interestRate         594068 non-null  float64
 3   installment          594068 non-null  float64
 4   grade                594068 non-null  int64  
 5   subGrade             594068 non-null  int64  
 6   employmentTitle      594068 non-null  float64
 7   employmentLength     594068 non-null  float64
 8   homeOwnership        594068 non-null  int64  
 9   annualIncome         594068 non-null  float64
 10  verificationStatus   594068 non-null  int64  
 11  isDefault            594068 non-null  int64  
 12  purpose              594068 non-null  int64  
 13  postCode             594068 non-null  float64
 14  regionCode           594068 non-null  int64  
 15  dti                  594068 non-null  float64
 16  delinquency_2years   594068 non-null  float64
 17  ficoRangeLow         594068 non-null  float64
 18  ficoRangeHigh        594068 non-null  float64
 19  openAcc              594068 non-null  float64
 20  pubRec               594068 non-null  float64
 21  pubRecBankruptcies   594068 non-null  float64
 22  revolBal             594068 non-null  float64
 23  revolUtil            594068 non-null  float64
 24  totalAcc             594068 non-null  float64
 25  initialListStatus    594068 non-null  int64  
 26  applicationType      594068 non-null  int64  
 27  title                594068 non-null  float64
 28  policyCode           594068 non-null  float64
 29  n0                   594068 non-null  float64
 30  n1                   594068 non-null  float64
 31  n2                   594068 non-null  float64
 32  n3                   594068 non-null  float64
 33  n4                   594068 non-null  float64
 34  n5                   594068 non-null  float64
 35  n6                   594068 non-null  float64
 36  n7                   594068 non-null  float64
 37  n8                   594068 non-null  float64
 38  n9                   594068 non-null  float64
 39  n10                  594068 non-null  float64
 40  n11                  594068 non-null  float64
 41  n12                  594068 non-null  float64
 42  n13                  594068 non-null  float64
 43  n14                  594068 non-null  float64
 44  issueDateDT          594068 non-null  int64  
 45  earliesCreditLineDT  594068 non-null  int64  
dtypes: float64(34), int64(12)
memory usage: 208.5 MB
# 相关系数选择特征
from sklearn.feature_selection import SelectKBest
from scipy.stats import pearsonr
#选择K个最好的特征，返回选择特征后的数据
#第一个参数为计算评估特征是否好的函数，该函数输入特征矩阵和目标向量，
#输出二元组（评分，P值）的数组，数组第i项为第i个特征的评分和P值。在此定义为计算相关系数
#参数k为选择的特征个数
​
SK = SelectKBest(f_classif,k=20).fit(train_data.drop(columns=['isDefault']),train_data['isDefault'])
SK.get_feature_names_out()
array(['loanAmnt', 'term', 'interestRate', 'installment', 'grade',
       'subGrade', 'homeOwnership', 'annualIncome', 'verificationStatus',
       'purpose', 'dti', 'ficoRangeLow', 'ficoRangeHigh', 'revolUtil',
       'n1', 'n2', 'n3', 'n9', 'n14', 'issueDateDT'], dtype=object)
corr_desc(train_data[SK.get_feature_names_out()])

模型
模型构建
xgb模型
def train_xgb(degree=1,booster='gbtree',n_estimators=10,max_depth=None,learning_rate=0.03,max_leaves=0,min_child_weight=1):
    pipe = Pipeline([
        ('poly_features',PolynomialFeatures(degree = degree, include_bias=False)),
        ('scaler', MinMaxScaler()), 
        ('XGBR', XGBClassifier(booster='gbtree'
                               ,random_state=7
                               ,verbosity=1
                               ,n_estimators =n_estimators 
                               ,max_depth=max_depth
                               ,max_leaves=max_leaves
                               ,learning_rate=learning_rate
                               ,min_child_weight=min_child_weight))
    ])
    return pipe
逻辑回归
def train_logistic_regression(degree=1,penalty='none',C=1.0,params={}):
    pipe = Pipeline([
        ('poly_features',PolynomialFeatures(degree = degree, include_bias=False)),
        ('scaler', MinMaxScaler()), 
        ('LR', LogisticRegression(penalty=penalty,tol=1e-4,C=C,max_iter=int(1e6)))])
    return pipe
支持向量机
def train_svm_Classifier(Xtrain,Xtest,Ytrain,Ytest,params={}):
    model = LinearSVC()
    model.fit(Xtrain,Ytrain)
    y_pred = model.predict(Xtest)
    score = roc_auc_score(Ytest,y_pred)
    print('Svm_Classifier AUC:',score,'Svm_Classifier score:',model.score(Xtest,Ytest))
    return score
随机森林
def train_RandomForest_Classifier(n_estimators=100,criterion='gini',max_depth=None,max_features='sqrt',min_samples_split=2,min_impurity_decrease=0):
    pipe = Pipeline([
        ('scaler', MinMaxScaler()), 
        ('RF', RandomForestClassifier(n_estimators=n_estimators
                                      ,criterion=criterion
                                      ,max_depth=max_depth
                                      ,min_samples_split=min_samples_split
                                      ,min_impurity_decrease=min_impurity_decrease))
    ])
    return pipe
最近邻
from sklearn.model_selection import train_test_split
Xtrain,Xtest,Ytrain,Ytest = train_test_split(train_data[SK.get_feature_names_out()]
                                             ,train_data['isDefault']
                                             ,stratify=train_data['isDefault']
                                             ,test_size=0.2)
模型交叉验证并调参
X = train_data[SK.get_feature_names_out()]
y = train_data['isDefault']
def cross_result(X,y,estimator,n_splits):
    # 交叉验证
    cv = KFold(n_splits=n_splits, shuffle=False,random_state=None)
    result = cross_validate(estimator
                             ,X,y
                             ,cv = cv
                             ,scoring="roc_auc"
                             ,return_train_score=True
                             ,verbose=True
                             )
    return result
# for i, (train_index, test_index) in enumerate(kf.split(X)):
#     print('************************************ {} *********************************'.format(str(i+1)))
#     Xtrain,Xtest,Ytrain,Ytest = X.iloc[train_index],X.iloc[test_index],y.iloc[train_index],y.iloc[test_index]
#     pipe = train_xgb(degree=2)
#     pipe.fit(Xtrain,Ytrain)
#     # model = train_xgb(Xtrain,Xtest,Ytrain,Ytest,params)
#     print("train_auc:",roc_auc_score(pipe.predict(Xtrain),Ytrain),"test_auc:",roc_auc_score(pipe.predict(Xtest),Ytest),"train_accuracy_score:",accuracy_score(pipe.predict(Xtrain),Ytrain),"test_accuracy_score:",accuracy_score(pipe.predict(Xtest),Ytest))
xgb交叉验证
xgb_model = train_xgb(degree=1,booster='gbtree',n_estimators=20,max_depth=11,learning_rate=0.03,max_leaves=0)
xgb_result = cross_result(X,y,xgb_model,5)
xgb_result
[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.
[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:  1.7min finished
{'fit_time': array([19.46292663, 19.16217995, 19.12600303, 19.08323622, 19.14989829]),
 'score_time': array([0.14918971, 0.16876912, 0.1535418 , 0.16472125, 0.15149832]),
 'test_score': array([0.71553794, 0.71191969, 0.71753529, 0.71412831, 0.71351067]),
 'train_score': array([0.75927729, 0.76062049, 0.75817444, 0.76050011, 0.75996534])}
RandomForest交叉验证
RF_model = train_RandomForest_Classifier(n_estimators=10
                                         ,max_depth=11
                                         ,max_features=10
                                         ,min_samples_split=5
                                         ,min_impurity_decrease=0)
RF_result = cross_result(X,y,RF_model,5)
RF_result
[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.
[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:   44.9s finished
{'fit_time': array([7.77110577, 8.01524329, 7.85951114, 7.81658816, 7.84747458]),
 'score_time': array([0.20302272, 0.2074759 , 0.20156932, 0.20274496, 0.20218325]),
 'test_score': array([0.7139858 , 0.71154043, 0.71697665, 0.71278119, 0.71132896]),
 'train_score': array([0.745549  , 0.74699029, 0.74508097, 0.7461957 , 0.74647574])}
逻辑回归交叉验证
LR_model = train_logistic_regression(degree=1)
LR_result = cross_result(X,y,LR_model,5)
LR_result
[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.
[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:  2.5min finished
{'fit_time': array([31.51272941, 25.17560077, 31.17797995, 30.77339959, 25.67836738]),
 'score_time': array([0.30646062, 0.30772758, 0.30553675, 0.30428767, 0.30517864]),
 'test_score': array([0.70997583, 0.7065469 , 0.71297466, 0.70898233, 0.70780735]),
 'train_score': array([0.70919827, 0.71000668, 0.70841244, 0.70944768, 0.70972491])}
params={'XGBR__n_estimators':[3,5,7],
        'XGBR__booster':['gbtree','gblinear'],
         'XGBR__learning_rate':[0.1,0.3],
         'XGBR__reg_alpha':[0,0.3,0.5,0.7],
         #'reg_lambda':0,
         'XGBR__max_depth':[3, 5, 6, 7, 9, 12],
         'XGBR__min_child_weight':[1, 3, 5, 7],
         #'colsample_bytree':1
        }
贝叶斯优化调参
!pip install hyperopt
Looking in indexes: https://mirrors.cloud.aliyuncs.com/pypi/simple/
Collecting hyperopt
  Downloading https://mirrors.cloud.aliyuncs.com/pypi/packages/b6/cd/5b3334d39276067f54618ce0d0b48ed69d91352fbf137468c7095170d0e5/hyperopt-0.2.7-py2.py3-none-any.whl (1.6 MB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.6/1.6 MB 7.8 MB/s eta 0:00:00a 0:00:010m
Requirement already satisfied: networkx>=2.2 in /opt/conda/lib/python3.7/site-packages (from hyperopt) (2.6.3)
Collecting cloudpickle
  Downloading https://mirrors.cloud.aliyuncs.com/pypi/packages/15/80/44286939ca215e88fa827b2aeb6fa3fd2b4a7af322485c7170d6f9fd96e0/cloudpickle-2.2.1-py3-none-any.whl (25 kB)
Requirement already satisfied: future in /opt/conda/lib/python3.7/site-packages (from hyperopt) (0.18.2)
Requirement already satisfied: scipy in /opt/conda/lib/python3.7/site-packages (from hyperopt) (1.7.3)
Requirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from hyperopt) (1.16.0)
Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from hyperopt) (1.18.5)
Collecting py4j
  Downloading https://mirrors.cloud.aliyuncs.com/pypi/packages/10/30/a58b32568f1623aaad7db22aa9eafc4c6c194b429ff35bdc55ca2726da47/py4j-0.10.9.7-py2.py3-none-any.whl (200 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 200.5/200.5 kB 4.7 MB/s eta 0:00:00a 0:00:01
Requirement already satisfied: tqdm in /opt/conda/lib/python3.7/site-packages (from hyperopt) (4.64.1)
Installing collected packages: py4j, cloudpickle, hyperopt
Successfully installed cloudpickle-2.2.1 hyperopt-0.2.7 py4j-0.10.9.7
WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv

[notice] A new release of pip available: 22.3.1 -> 24.0
[notice] To update, run: pip install --upgrade pip
from hyperopt import hp, fmin, tpe, Trials, partial
from hyperopt.early_stop import no_progress_loss
​
def hyperopt_objective(params):
    xgb = train_xgb(degree=1,
                    booster='gbtree'
                    ,n_estimators=int(params['n_estimators'])
                    ,max_depth=int(params['max_depth'])
                    ,learning_rate=params['learning_rate']
                    ,min_child_weight=int(params['min_child_weight']))
    #交叉验证结果，auc分数
    cv = KFold(n_splits=5, shuffle=True,random_state=7)
    validation_loss = cross_validate(xgb
                             ,X,y
                             ,cv = cv
                             ,scoring="roc_auc"
                             ,return_train_score=True
                             ,verbose=False
                             ,error_score='raise'
                             )
    # 由于默认找最小值，因此把auc分数前加个负号
    return -np.mean(validation_loss['test_score'])
params = {'n_estimators':hp.quniform('n_estimators',5,50,2)
         ,'max_depth':hp.quniform('max_depth',5,15,1)
         ,'learning_rate':hp.quniform('learning_rate',0,0.3,0.03)
         ,'min_child_weight':hp.quniform('min_child_weight',1,5,1)
         }
def param_hyperopt(max_evals=100):
    
    # 保存迭代过程
    trials = Trials()
    
    #设置提前停止
    early_stop_fn = no_progress_loss(5)
    
    # 设置代理模型
    params_best = fmin(hyperopt_objective
                      ,space=params
                      ,algo = tpe.suggest
                      ,max_evals=max_evals
                      ,verbose=True
                      ,trials = trials
                      ,early_stop_fn = early_stop_fn)
    #打印最优参数，fmin会自动打印最佳分数
    print("\n","\n","best params: ", params_best,
          "\n")
    return params_best, trials
params_best, trials = param_hyperopt(30)
100%|██████████| 30/30 [56:53<00:00, 113.78s/trial, best loss: -0.7225354435929108] 

 
 best params:  {'learning_rate': 0.18, 'max_depth': 6.0, 'min_child_weight': 5.0, 'n_estimators': 50.0} 

模型保存
Logistic Regression
model = train_logistic_regression(degree=1)
model.set_params(LR__solver='saga')
model.set_params(LR__penalty='l2')
model.fit(train_data[SK.get_feature_names_out()]
        ,train_data['isDefault'])
joblib.dump(model,'Logistic_regression.model')
['Logistic_regression.model']
XGBC
model = train_xgb(degree=1,booster='gbtree'
                  ,n_estimators=50
                  ,max_depth=6
                  ,min_child_weight=5
                  ,learning_rate=0.18
                  ,max_leaves=0)
model.fit(train_data[SK.get_feature_names_out()]
        ,train_data['isDefault'])
joblib.dump(model,'XGBC.model')
['XGBC.model']
RandomForest
model = train_RandomForest_Classifier(n_estimators=10
                                         ,max_depth=11
                                         ,max_features=10
                                         ,min_samples_split=5
                                         ,min_impurity_decrease=0)
model.fit(train_data[SK.get_feature_names_out()]
        ,train_data['isDefault'])
joblib.dump(model,'RF.model')
['RF.model']
模型预测
SK.get_feature_names_out()
array(['loanAmnt', 'term', 'interestRate', 'installment', 'grade',
       'subGrade', 'homeOwnership', 'annualIncome', 'verificationStatus',
       'purpose', 'dti', 'ficoRangeLow', 'ficoRangeHigh', 'revolUtil',
       'n1', 'n2', 'n3', 'n9', 'n14', 'issueDateDT'], dtype=object)
test_data = pd.read_csv('/mnt/workspace/downloads/168012/testA.csv')
# 构建特征
test_data['issueDate'] = pd.to_datetime(test_data['issueDate'],format='%Y-%m-%d')
startdate = datetime.datetime.strptime('2007-06-01', '%Y-%m-%d')
test_data['issueDateDT'] = test_data['issueDate'].apply(lambda x: x-startdate).dt.days
# 筛选数据
test_data_pred = test_data[SK.get_feature_names_out()]
test_data_pred['grade'] = deal_grade(test_data_pred['grade'])
test_data_pred['subGrade'] = deal_subgrade(test_data_pred['subGrade'])
​
# 补充缺失值
test_data_pred = test_data_pred.fillna(test_data_pred.median())
模型融合
from tqdm import tqdm
​
result = pd.DataFrame()
result['id'] = test_data['id']
​
model_name = ['RF','Logistic_regression','XGBC']
​
for i in tqdm(model_name):
    pred_model = joblib.load(i+'.model')
    result[i+'_PRE'] = pred_model.predict_proba(test_data_pred)[:,1]
100%|██████████| 3/3 [00:00<00:00,  4.11it/s]
result
id	RF_PRE	Logistic_regression_PRE	XGBC_PRE
0	800000	0.101910	0.081135	0.319227
1	800001	0.314904	0.304737	0.418891
2	800002	0.317615	0.359463	0.454282
3	800003	0.297796	0.263870	0.421738
4	800004	0.272894	0.358186	0.443214
...	...	...	...	...
199995	999995	0.123374	0.137986	0.331959
199996	999996	0.047522	0.072072	0.291659
199997	999997	0.364447	0.308497	0.344575
199998	999998	0.270071	0.143330	0.406798
199999	999999	0.043524	0.060592	0.293907
200000 rows × 4 columns

result['isDefault'] = result['RF_PRE']*0.4+result['Logistic_regression_PRE']*0.2+result['XGBC_PRE']*0.4
result
id	RF_PRE	Logistic_regression_PRE	XGBC_PRE	isDefault
0	800000	0.101910	0.081135	0.319227	0.184682
1	800001	0.314904	0.304737	0.418891	0.354465
2	800002	0.317615	0.359463	0.454282	0.380652
3	800003	0.297796	0.263870	0.421738	0.340587
4	800004	0.272894	0.358186	0.443214	0.358080
...	...	...	...	...	...
199995	999995	0.123374	0.137986	0.331959	0.209730
199996	999996	0.047522	0.072072	0.291659	0.150087
199997	999997	0.364447	0.308497	0.344575	0.345308
199998	999998	0.270071	0.143330	0.406798	0.299413
199999	999999	0.043524	0.060592	0.293907	0.147091
200000 rows × 5 columns

结果输出
result[['id','isDefault']].to_csv('sample_submit.csv',index=False)